{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the model before attempting to run this NB\n",
    "# python -m spacy download en_core_web_lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import seaborn as sns\n",
    "import spacy\n",
    "from spacy.gold import GoldCorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_dir = '../data/en/combined/spacy_formatted/'\n",
    "devset = 'spacy_dev_combined_std.json'\n",
    "dev_path = os.path.join(file_dir, devset)\n",
    "\n",
    "# Create a temp file (devset but with '_2' appended) and\n",
    "# replace all the 'PER' tags with 'PERSON' to match spacy NER model\n",
    "dev_path_2 = dev_path[:-5] + '_2' + dev_path[-5:]\n",
    "\n",
    "with open(dev_path) as f:\n",
    "    with open(dev_path_2, 'w') as f2:\n",
    "        f2.write(f.read().replace('-PER', '-PERSON'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_lg')\n",
    "gc = GoldCorpus(dev_path_2, dev_path_2)\n",
    "docs = gc.train_docs(nlp)\n",
    "scorer = nlp.evaluate(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAJOCAYAAAB4PjmuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de5Rld13n/ffHNCHhmlslT987ShOJuuhgEaMIjEnIAm/p9TwJBG8djdPO80SBAcQo+IDPgIJLZRQHnV5Epgch5GKYZJgRDT2AoCGkcsHcwMaQ9JWkAgkQiEDi9/nj7Maypyt1flXn1DlV/X6tVeuc/du/fc737MC3P7X3rn1SVUiSJKl/3zHqAiRJkpYaA5QkSVIjA5QkSVIjA5QkSVIjA5QkSVIjA5QkSVIjA5QkSVIjA5QOKck9SR5J8nCS+5K8O8lTRl2XJA1S1+vOHnUdWnoMUHo8P1FVTwGeAzwXeEPLxklWDKUqSZJGzAClOVXVXuAvge9N8vQklybZn2RvkjcnOQIgyYVJ/jbJ25N8CXhTkmck+ViSLyd5IMnlB143yQ8lubFbd2OSH5qx7qNJ/kP3el9N8tdJTlj0Dy/psJTk3yb5XJIvJbk2yaoZ674nyXXduvuS/MYoa9VoGKA0pyRrgR8FbgG2A48CzwBOA84BfnHG9B8A7gZOBN4C/Afgr4FjgTXAO7rXPA74H8AfAccDfwD8jyTHz3itnwJ+vnutI4HXDuUDStIMSc4Efgd4KbASuBd4f7fuqcCHgQ8Bq+j1wh2jqVSjZIDS4/lvSR4CPgF8DHgX8BLgVVX1taq6H3g7cMGMbfZV1Tuq6tGqegT4FrAeWFVV/1RVn+jm/Riws6re0829DPgM8BMzXuvdVfUP3etcAWwa5oeVpM5PA39WVTdX1TeAXwd+MMkG4MeBL1TV73c97atVdcMIa9WIGKD0eDZX1TFVtb6q/h/gJOAJwP4kD3Xh6j/TO0J0wO6DXuN1QIBPJbkjyS9046vo/VY3073A6hnLX5jx/OuAF7FLWgz/qj9V1cPAF+n1p7XAP46oLo0RL/JVi93AN4ATqurRWebUv1qo+gLwbwGS/DDw4SR/A+yjd2RqpnX0DotL0ij9q/6U5Mn0LjXYS68PvnxEdWmMeARKfauq/fSuZ/r9JE9L8h1JvivJC2fbJsn5SdZ0iw/SC1iPAf8TeGaSn0qyIsnLgFOBDw75Y0jSwZ6Q5KgDP/QuGfj5JJuSPBH4beCGqrqHXo/6P5K8KskTkzw1yQ+MsHaNiAFKrX6O3gXdd9ILRFfRu8hyNs8FbkjyMHAt8Mqq+nxVfZHetQSvoXdo/HXAj1fVA8MsXpIO4X8Cj8z4eT7wm8BfAPuB76K71rOqvgq8iN71ml8AdgI/svgla9RSVXPPkiRJ0rd5BEqSJKmRAUqSJKmRAUqSJKmRAUqSJKnRot4H6oQTTqgNGzYs5ltKGqGbbrrpgaqaGHUdg2D/kg4/j9fDFjVAbdiwgampqcV8S0kjlOTgu80vWfYv6fDzeD3MU3iSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNFvWrXOayfv16du3aNeoyxsa6deu4995l800YkiQtG2MVoHbt2sUt+z416jLGxmmrTh91CZIk6RA8hSdJktTIACVJktSorwCV5N8nuSPJ7UkuS3JUkpOT3JBkZ5LLkxw57GIlSZLGwZwBKslq4BXAZFV9L3AEcAHwNuDtVbUReBC4aJiFSpIkjYt+T+GtAI5OsgJ4ErAfOBO4qlu/Hdg8+PIkSZLGz5wBqqr2Ar8H7KIXnL4M3AQ8VFWPdtP2AKsPtX2SrUmmkkxNT08PpmpJWgT2L0mz6ecU3rHAucDJwCrgycBLDjG1DrV9VW2rqsmqmpyYmFhIrZK0qOxfkmbTzym8s4HPV9V0VX0LuBr4IeCY7pQewBpg35BqlCRJGiv9BKhdwBlJnpQkwFnAncBHgPO6OVuAa4ZToiRJ0njp5xqoG+hdLH4zcFu3zTbg14BXJ/kccDxw6RDrlCRJGht9fZVLVb0ReONBw3cDfteIJEk67HgnckmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEZ93QdKUs+qNevYv3f3qMsYGytXr2Xfnl2jLkOSFp0BSmqwf+9unv+WK0ddxtj4+OvPH3UJkjQSnsKTJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqNGeASnJKkltn/HwlyauSHJfkuiQ7u8djF6NgSZKkUZszQFXVZ6tqU1VtAr4f+DrwAeASYEdVbQR2dMuSJEnLXuspvLOAf6yqe4Fzge3d+HZg8yALkyRJGletAeoC4LLu+UlVtR+gezzxUBsk2ZpkKsnU9PT0/CuVpEVm/5I0m74DVJIjgZ8Ermx5g6raVlWTVTU5MTHRWp8WaM26dSTxJ2HNunWj/s+hJcb+JWk2KxrmvgS4uaru65bvS7KyqvYnWQncP/jytFB7d+/mzR/78KjLGAtveOHZoy5BkrRMtJzCezn/cvoO4FpgS/d8C3DNoIqSJEkaZ30FqCRPAl4EXD1j+K3Ai5Ls7Na9dfDlSZIkjZ++TuFV1deB4w8a+yK9v8qTJEk6rHgnckmSpEYGKEmSpEYGKEmSpEYttzGQJD2O9evXs2vXrlGXMRbWrVvHvffeO+oypKExQEnSgOzatYtb9n1q1GWMhdNWnT7qEqSh8hSeJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSo74CVJJjklyV5DNJ7kryg0mOS3Jdkp3d47HDLlaSJGkc9HsE6g+BD1XVdwPPBu4CLgF2VNVGYEe3LEmStOzNGaCSPA14AXApQFV9s6oeAs4FtnfTtgObh1WkJEnSOOnnCNR3AtPAu5PckuRdSZ4MnFRV+wG6xxMPtXGSrUmmkkxNT08PrHBJGjb7l6TZ9BOgVgDPAf6kqk4DvkbD6bqq2lZVk1U1OTExMc8yJWnx2b8kzaafALUH2FNVN3TLV9ELVPclWQnQPd4/nBIlSZLGy5wBqqq+AOxOcko3dBZwJ3AtsKUb2wJcM5QKJUmSxsyKPuf9CvDeJEcCdwM/Ty98XZHkImAXcP5wSpQkSRovfQWoqroVmDzEqrMGW44kSdL4807kkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjVb0MynJPcBXgceAR6tqMslxwOXABuAe4KVV9eBwypQkSRofLUegfqSqNlXVZLd8CbCjqjYCO7plSZKkZW8hp/DOBbZ3z7cDmxdejiRJ0vjrN0AV8NdJbkqytRs7qar2A3SPJx5qwyRbk0wlmZqenl54xZK0SOxfkmbTb4B6XlU9B3gJcHGSF/T7BlW1raomq2pyYmJiXkVK0ijYvyTNpq8AVVX7usf7gQ8ApwP3JVkJ0D3eP6wiJUmSxsmcASrJk5M89cBz4BzgduBaYEs3bQtwzbCKlCRJGif93MbgJOADSQ7Mf19VfSjJjcAVSS4CdgHnD69MSZKk8TFngKqqu4FnH2L8i8BZwyhKkiRpnHknckmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEZ9B6gkRyS5JckHu+WTk9yQZGeSy5McObwyJUmSxkfLEahXAnfNWH4b8Paq2gg8CFw0yMIkSZLGVV8BKska4MeAd3XLAc4EruqmbAc2D6NASZKkcdPvEaj/CLwO+Odu+Xjgoap6tFveA6w+1IZJtiaZSjI1PT29oGIlaTHZvyTNZs4AleTHgfur6qaZw4eYWofavqq2VdVkVU1OTEzMs0xJWnz2L0mzWdHHnOcBP5nkR4GjgKfROyJ1TJIV3VGoNcC+4ZUpSZI0PuY8AlVVv15Va6pqA3AB8L+q6qeBjwDnddO2ANcMrUpJkqQxspD7QP0a8Ookn6N3TdSlgylJkiRpvPVzCu/bquqjwEe753cDpw++JEmSpPHmncglSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIazRmgkhyV5FNJPp3kjiS/1Y2fnOSGJDuTXJ7kyOGXK0mSNHr9HIH6BnBmVT0b2AS8OMkZwNuAt1fVRuBB4KLhlSlJkjQ+5gxQ1fNwt/iE7qeAM4GruvHtwOahVChJkjRm+roGKskRSW4F7geuA/4ReKiqHu2m7AFWz7Lt1iRTSaamp6cHUbMkLQr7l6TZ9BWgquqxqtoErAFOB551qGmzbLutqiaranJiYmL+lUrSIrN/SZpN01/hVdVDwEeBM4BjkqzoVq0B9g22NEmSpPHUz1/hTSQ5pnt+NHA2cBfwEeC8btoW4JphFSlJkjROVsw9hZXA9iRH0AtcV1TVB5PcCbw/yZuBW4BLh1inJEnS2JgzQFXV3wOnHWL8bnrXQ0mSJB1WvBO5JElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSozkDVJK1ST6S5K4kdyR5ZTd+XJLrkuzsHo8dfrmSJEmj188RqEeB11TVs4AzgIuTnApcAuyoqo3Ajm5ZkiRp2ZszQFXV/qq6uXv+VeAuYDVwLrC9m7Yd2DysIiVJksZJ0zVQSTYApwE3ACdV1X7ohSzgxFm22ZpkKsnU9PT0wqqVpEVk/5I0m74DVJKnAH8BvKqqvtLvdlW1raomq2pyYmJiPjVK0kjYvyTNpq8AleQJ9MLTe6vq6m74viQru/UrgfuHU6IkSdJ46eev8AJcCtxVVX8wY9W1wJbu+RbgmsGXJ0mSNH5W9DHnecDPArclubUb+w3grcAVSS4CdgHnD6dESZKk8TJngKqqTwCZZfVZgy1HkiRp/HknckmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEZzBqgkf5bk/iS3zxg7Lsl1SXZ2j8cOt0xJkqTx0c8RqP8CvPigsUuAHVW1EdjRLUuSJB0W5gxQVfU3wJcOGj4X2N493w5sHnBdkiRJY2u+10CdVFX7AbrHE2ebmGRrkqkkU9PT0/N8O0lafPYvSbMZ+kXkVbWtqiaranJiYmLYbydJA2P/kjSb+Qao+5KsBOge7x9cSZIkSeNtvgHqWmBL93wLcM1gypEkSRp//dzG4DLgeuCUJHuSXAS8FXhRkp3Ai7plSZKkw8KKuSZU1ctnWXXWgGuRJElaErwTuSRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUqM5v8pFkiQtD6vWrGP/3t2jLmMsrFy9ln17ds17ewOUJEmHif17d/P8t1w56jLGwsdff/6CtvcUniRJUiMDlCRJUiMDlCRJUiMDlCRJUiMvIpckjaU169axd7d/MXbA6rVr2bNr/n81psEyQEmSxtLe3bt588c+POoyxsYbXnj2qEvQDJ7CkyRJarSgAJXkxUk+m+RzSS4ZVFGSJEnjbN4BKskRwH8CXgKcCrw8yamDKkySJGlcLeQI1OnA56rq7qr6JvB+4NzBlCVJkjS+UlXz2zA5D3hxVf1it/yzwA9U1S8fNG8rsLVbPAX47PzLXTQnAA+Muohlxn06WEtlf66vqolRFzFf9i913KeDt1T26aw9bCF/hZdDjP1vaayqtgHbFvA+iy7JVFVNjrqO5cR9Oljuz8Vh/xK4T4dhOezThZzC2wOsnbG8Bti3sHIkSZLG30IC1I3AxiQnJzkSuAC4djBlSZIkja95n8KrqkeT/DLwV8ARwJ9V1R0Dq2y0ltQh+yXCfTpY7k/Nxv9tDJ77dPCW/D6d90XkkiRJhyvvRC5JktTIACVJktRoyQeoJI8luTXJ7UmuTPKkAbzmZJI/epz1q5JctdD3WWoO2tf/PckxA379C5P8cff8TUleO8jXX6pm7PcDPxuSHJ/kI0kePrDPtPTYvxaXPWzxLef+teQDFPBIVW2qqu8Fvgn8u5kr09P0Oatqqqpe8Tjr91XVefMrd0mbua+/BFw86oIOEwf2+4Gfe4B/An4TOOwb9BJn/1pc9rDFt2z713IIUDN9HHhGl3DvSvJO4GZgbZJzklyf5ObuN72nACR5bpK/S/LpJJ9K8tQk/ybJB7v1L5yRnG/p1m9Icnu3/qgk705yW7f+R7rxC5NcneRDSXYm+d0R7ZNhuR5YfWAhya8muTHJ3yf5rRnjP9eNfTrJe7qxn0hyQ7e/PpzkpBHUv6RV1deq6hP0GpGWB/vX4rKHjchy6V/LJkAlWUHvi41v64ZOAf5rVZ0GfA14A3B2VT0HmAJend79qy4HXllVzwbOBh456KVfC1xcVZuA5x9i/cUAVfV9wMuB7UmO6tZtAl4GfB/wsiRrWQbS+yLps+ju+5XkHGAjve9H3AR8f5IXJPke4PXAmd3+fWX3Ep8Azuj+27wfeN0if4Sl5ugZ/wh+YNTFaPDsX4vLHraolm3/WshXuYyLo5Pc2j3/OHApsAq4t6o+2Y2fAZwK/G0SgCPp/fZxCrC/qm4EqKqvAHRzDvhb4A+SvBe4uqr2HLT+h4F3dNt/Jsm9wDO7dTuq6svda94JrAd2D+hzj8KBfb0BuAm4rhs/p/u5pVt+Cr1m9Gzgqqp6AKCqvtStXwNcnmQlvf8Wn1+U6peuR7p/ALX82L8Wlz1s8S3b/rUcjkDNPL/6K1X1zW78azPmBLhuxrxTq+qibvxxb4RVVW8FfhE4Gvhkku8+aMqhvhPwgG/MeP4YSz+wHvg/wnp6TePA9QMBfmfG/n1GVV3K7Pv3HcAfd7/1/hJw1CHmSIcD+9fisodpYJZDgOrHJ4HnJXkGQJInJXkm8BlgVZLnduNP7Q6lf1uS76qq26rqbfQOnR/cgP4G+Olu7jOBdSyNb2yft+630lcAr03yBHp3o/+FGddlrE5yIrADeGmS47vx47qXeDqwt3u+ZVGLl5Ye+9eA2cM0CMvhN4o5VdV0kguBy5I8sRt+Q1X9Q5KXAe9IcjS96wPOPmjzV3UXVj4G3An8JbByxvp3An+a5DbgUeDCqvrGQYfJl52quiXJp4ELquo9SZ4FXN997oeBn6mqO5K8BfhYksfoHR6/EHgTcGWSvfT+cTh5FJ9hqUtyD/A04Mgkm4FzqurO0ValQbN/DYc9bLSWQ//yq1wkSZIaHS6n8CRJkgbGACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJktTIACVJ0pAkeVOSP++eb0hSSVaMui4tnAFKC5bkwiS3Jfl6ki8k+ZMkx4y6LkmHL/uShs0ApQVJ8hrgbcCvAk8HzgDWA9clOfIQ8/3NS9JQtfalBbyP/ewwZoDSvCV5GvBbwK9U1Yeq6ltVdQ/wUnrN6me6w9dXJfnzJF8BLkxydJLtSR5McleS1yXZM8KPImmZ6KMvvTbJI0mOm7HNaUkeSPKEbvkXut70YJK/SrJ+xtxKcnGSncDObuwPk+xO8pUkNyV5/iJ+ZI2IAUoL8UPAUcDVMwer6mHgL4EXdUPnAlcBxwDvBd4IbAC+s5vzM4tTrqTDwFx96fuA64H/a8bqnwKuqqpvJdkM/AbwfwITwMeByw56j83ADwCndss3ApuA44D3AVcmOWqAn0ljyAClhTgBeKCqHj3Euv3deoDrq+q/VdU/V9Uj9H4T/O2qerCq9gB/tEj1Slr++ulL7wNeDpAkwAXdGMAvAb9TVXd1r/HbwKaZR6G69V/q+hlV9edV9cWqerSqfh94InDKMD6cxocBSgvxAHDCLNcBrOzWA+w+aN2qg8YOXi9J89VPX7oK+MEkq4AXAEXvSBP0TvP9YZKHkjwEfAkIsHrG6/yrnpXkNd0pvy932zydf/kFUsuUAUoLcT3wDXqHur8tyZOBlwA7uqE6aLv9wJoZy2uHVaCkw86cfamqHgL+mt7R8J8CLquqA31qN/BLVXXMjJ+jq+rvZrxczXjd5wO/1r3WsVV1DPBleqFLy5gBSvNWVV+md7HmO5K8OMkTkmwArgT2AO+ZZdMrgF9PcmyS1cAvL0a9kpa/hr70PuDn6F0L9b4ZL/Gn9PrT9wAkeXqS8x/nLZ8KPApMAyuS/L/A0wb3iTSuDFBakKr6XXoXXP4e8BXgBnq/wZ1VVd+YZbP/j14j+zzwYXqH02ebK0lN+uxL1wIbgfuq6tMztv0AvVsgvL/7y+Hb6R25ms1f0bs4/R+Ae4F/wssSDgv5l6OW0mgk+b+BC6rqhaOuRZKkfngESosuycokz0vyHUlOAV4DfGDUdUmS1C/voqpROBL4z8DJwEPA+4F3jrQiSZIaeApPkiSpkafwJEmSGi3qKbwTTjihNmzYsJhvKWmEbrrppgeqamLUdQyC/Us6/DxeD1vUALVhwwampqYW8y0ljVCSe0ddw6DYv6TDz+P1ME/hSZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNeorQCX590nuSHJ7ksuSHJXk5CQ3JNmZ5PIkRw67WEmSpHEwZ4BKshp4BTBZVd8LHAFcALwNeHtVbQQeBC4aZqGSJEnjot9TeCuAo5OsAJ4E7AfOBK7q1m8HNg++PEmSpPEzZ4Cqqr3A7wG76AWnLwM3AQ9V1aPdtD3A6kNtn2RrkqkkU9PT04OpWpIWgf1L0mz6OYV3LHAucDKwCngy8JJDTK1DbV9V26pqsqomJyYmFlKrJC0q+5ek2fRzCu9s4PNVNV1V3wKuBn4IOKY7pQewBtg3pBolSZLGSj8BahdwRpInJQlwFnAn8BHgvG7OFuCa4ZQoSZI0Xvq5BuoGeheL3wzc1m2zDfg14NVJPgccD1w6xDolSZLGxoq5p0BVvRF440HDdwOnD7wiSZKkMeedyCVJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhrNGaCSnJLk1hk/X0nyqiTHJbkuyc7u8djFKFiSJGnU5gxQVfXZqtpUVZuA7we+DnwAuATYUVUbgR3dsiRJ0rLXegrvLOAfq+pe4Fxgeze+Hdg8yMIkSZLGVWuAugC4rHt+UlXtB+geTzzUBkm2JplKMjU9PT3/SiVpkdm/JM2m7wCV5EjgJ4ErW96gqrZV1WRVTU5MTLTWJ0kjY/+SNJuWI1AvAW6uqvu65fuSrAToHu8fdHGSJEnjqCVAvZx/OX0HcC2wpXu+BbhmUEVJkiSNs74CVJInAS8Crp4x/FbgRUl2duveOvjyJEmSxs+KfiZV1deB4w8a+yK9v8qTJEk6rHgnckmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEZ9BagkxyS5KslnktyV5AeTHJfkuiQ7u8djh12sJEnSOOj3CNQfAh+qqu8Gng3cBVwC7KiqjcCOblmSJGnZmzNAJXka8ALgUoCq+mZVPQScC2zvpm0HNg+rSEmSpHHSzxGo7wSmgXcnuSXJu5I8GTipqvYDdI8nHmrjJFuTTCWZmp6eHljhkjRs9i9Js+knQK0AngP8SVWdBnyNhtN1VbWtqiaranJiYmKeZUrS4rN/SZpNPwFqD7Cnqm7olq+iF6juS7ISoHu8fzglSpIkjZc5A1RVfQHYneSUbugs4E7gWmBLN7YFuGYoFUqSJI2ZFX3O+xXgvUmOBO4Gfp5e+LoiyUXALuD84ZQoSZI0XvoKUFV1KzB5iFVnDbYcSZKk8eedyCVJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhoZoCRJkhqt6GdSknuArwKPAY9W1WSS44DLgQ3APcBLq+rB4ZQpSZI0PlqOQP1IVW2qqslu+RJgR1VtBHZ0y5IkScveQk7hnQts755vBzYvvBxJkqTx12+AKuCvk9yUZGs3dlJV7QfoHk881IZJtiaZSjI1PT298IolaZHYvyTNpt8A9byqeg7wEuDiJC/o9w2qaltVTVbV5MTExJgWSk4AAA0SSURBVLyKlKRRsH9Jmk1fAaqq9nWP9wMfAE4H7kuyEqB7vH9YRUqSJI2TOQNUkicneeqB58A5wO3AtcCWbtoW4JphFSlJkjRO+rmNwUnAB5IcmP++qvpQkhuBK5JcBOwCzh9emZIkSeNjzgBVVXcDzz7E+BeBs4ZRlCRJ0jjzTuSSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmNDFCSJEmN+g5QSY5IckuSD3bLJye5IcnOJJcnOXJ4ZUqSJI2PliNQrwTumrH8NuDtVbUReBC4aJCFSZIkjau+AlSSNcCPAe/qlgOcCVzVTdkObB5GgZIkSeOm3yNQ/xF4HfDP3fLxwENV9Wi3vAdYfagNk2xNMpVkanp6ekHFStJisn9Jms2cASrJjwP3V9VNM4cPMbUOtX1VbauqyaqanJiYmGeZkrT47F+SZrOijznPA34yyY8CRwFPo3dE6pgkK7qjUGuAfcMrU5IkaXzMeQSqqn69qtZU1QbgAuB/VdVPAx8BzuumbQGuGVqVkiRJY2Qh94H6NeDVST5H75qoSxdazPr160niT/ezfv36he5SSZI0BP2cwvu2qvoo8NHu+d3A6YMsZteuXdyy71ODfMkl7bRVA929kiRpQLwTuSRJUiMDlCRJUiMDlCRJUqOma6Ckw92qNevYv3f3qMsYGytXr2Xfnl2jLkOSFp0BSmqwf+9unv+WK0ddxtj4+OvPH3UJkjQSnsKTJElqZICSJElq5Cm8ZW7NunXs3e01OwCr165lzy6v15EkLZwBapnbu3s3b/7Yh0ddxlh4wwvPHnUJkqRlwlN4kiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjbyRpiRJh4lVa9axf6/fTgGwcvVa9u2Z/7dTGKAkSTpM7N+7m+e/5cpRlzEWPv768xe0/Zyn8JIcleRTST6d5I4kv9WNn5zkhiQ7k1ye5MgFVSJJkrRE9HMN1DeAM6vq2cAm4MVJzgDeBry9qjYCDwIXDa9MSZKk8TFngKqeh7vFJ3Q/BZwJXNWNbwc2D6VCSZKkMdPXX+ElOSLJrcD9wHXAPwIPVdWj3ZQ9wOpZtt2aZCrJ1PT09CBqlqRFYf+SNJu+AlRVPVZVm4A1wOnAsw41bZZtt1XVZFVNTkxMzL9SSVpk9i9Js2m6D1RVPQR8FDgDOCbJgb/iWwPsG2xpkiRJ46mfv8KbSHJM9/xo4GzgLuAjwHndtC3ANcMqUpIkaZz0cx+olcD2JEfQC1xXVNUHk9wJvD/Jm4FbgEuHWKckSdLYmDNAVdXfA6cdYvxuetdDSZIkHVb8LjxJkqRGBihJkqRGBihJkqRGBihJkqRGBihJkqRGBihJkqRG/dwHSpLUh/Xr17Nr165RlzEW1q1bx7333jvqMqShMUBJ0oDs2rWLW/Z9atRljIXTVnmbQC1vnsKTJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElq5J3IJUljac26dezdvXvUZYyN1WvXssevChobBihJ0ljau3s3b/7Yh0ddxth4wwvPHnUJmsFTeJIkSY3mDFBJ1ib5SJK7ktyR5JXd+HFJrkuys3s8dvjlSpIkjV4/R6AeBV5TVc8CzgAuTnIqcAmwo6o2Aju6ZUmSpGVvzgBVVfur6ubu+VeBu4DVwLnA9m7admDzsIqUJEkaJ03XQCXZAJwG3ACcVFX7oReygBNn2WZrkqkkU9PT0wurVpIWkf1L0mz6DlBJngL8BfCqqvpKv9tV1baqmqyqyYmJifnUKEkjYf+SNJu+AlSSJ9ALT++tqqu74fuSrOzWrwTuH06JkiRJ46Wfv8ILcClwV1X9wYxV1wJbuudbgGsGX54kSdL46edGms8Dfha4Lcmt3dhvAG8FrkhyEbALOH84JUqSJI2XOQNUVX0CyCyrzxpsOZIkSePPO5FLkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1mjNAJfmzJPcnuX3G2HFJrkuys3s8drhlSpIkjY9+jkD9F+DFB41dAuyoqo3Ajm5ZkiTpsDBngKqqvwG+dNDwucD27vl2YPOA65IkSRpb870G6qSq2g/QPZ4428QkW5NMJZmanp6e59tJ0uKzf0mazdAvIq+qbVU1WVWTExMTw347SRoY+5ek2cw3QN2XZCVA93j/4EqSJEkab/MNUNcCW7rnW4BrBlOOJEnS+OvnNgaXAdcDpyTZk+Qi4K3Ai5LsBF7ULUuSJB0WVsw1oapePsuqswZciyRJ0pLgncglSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaLShAJXlxks8m+VySSwZVlCRJ0jibd4BKcgTwn4CXAKcCL09y6qAKkyRJGlcLOQJ1OvC5qrq7qr4JvB84dzBlSZIkja9U1fw2TM4DXlxVv9gt/yzwA1X1ywfN2wps7RZPAT47/3IXzQnAA6MuYplxnw7WUtmf66tqYtRFzJf9Sx336eAtlX06aw9bsYAXzSHG/rc0VlXbgG0LeJ9Fl2SqqiZHXcdy4j4dLPfn4rB/Cdynw7Ac9ulCTuHtAdbOWF4D7FtYOZIkSeNvIQHqRmBjkpOTHAlcAFw7mLIkSZLG17xP4VXVo0l+Gfgr4Ajgz6rqjoFVNlpL6pD9EuE+HSz3p2bj/zYGz306eEt+n877InJJkqTDlXcilyRJamSAkiRJarTkA1SSx5LcmuT2JFcmedIAXnMyyR89zvpVSa5a6PssNQft6/+e5JgBv/6FSf64e/6mJK8d5OsvVTP2+4GfDUmOT/KRJA8f2Gdaeuxfi8setviWc/9a8gEKeKSqNlXV9wLfBP7dzJXpafqcVTVVVa94nPX7quq8+ZW7pM3c118CLh51QYeJA/v9wM89wD8Bvwkc9g16ibN/LS572OJbtv1rOQSomT4OPKNLuHcleSdwM7A2yTlJrk9yc/eb3lMAkjw3yd8l+XSSTyV5apJ/k+SD3foXzkjOt3TrNyS5vVt/VJJ3J7mtW/8j3fiFSa5O8qEkO5P87oj2ybBcD6w+sJDkV5PcmOTvk/zWjPGf68Y+neQ93dhPJLmh218fTnLSCOpf0qrqa1X1CXqNSMuD/Wtx2cNGZLn0r2UToJKsoPfFxrd1Q6cA/7WqTgO+BrwBOLuqngNMAa9O7/5VlwOvrKpnA2cDjxz00q8FLq6qTcDzD7H+YoCq+j7g5cD2JEd16zYBLwO+D3hZkrUsA+l9kfRZdPf9SnIOsJHe9yNuAr4/yQuSfA/weuDMbv++snuJTwBndP9t3g+8bpE/wlJz9Ix/BD8w6mI0ePavxWUPW1TLtn8t5KtcxsXRSW7tnn8cuBRYBdxbVZ/sxs8ATgX+NgnAkfR++zgF2F9VNwJU1VcAujkH/C3wB0neC1xdVXsOWv/DwDu67T+T5F7gmd26HVX15e417wTWA7sH9LlH4cC+3gDcBFzXjZ/T/dzSLT+FXjN6NnBVVT0AUFVf6tavAS5PspLef4vPL0r1S9cj3T+AWn7sX4vLHrb4lm3/Wg5HoGaeX/2VqvpmN/61GXMCXDdj3qlVdVE3/rg3wqqqtwK/CBwNfDLJdx805VDfCXjAN2Y8f4ylH1gP/B9hPb2mceD6gQC/M2P/PqOqLmX2/fsO4I+733p/CTjqEHOkw4H9a3HZwzQwyyFA9eOTwPOSPAMgyZOSPBP4DLAqyXO78ad2h9K/Lcl3VdVtVfU2eofOD25AfwP8dDf3mcA6lsY3ts9b91vpK4DXJnkCvbvR/8KM6zJWJzkR2AG8NMnx3fhx3Us8HdjbPd+yqMVLS4/9a8DsYRqE5fAbxZyqajrJhcBlSZ7YDb+hqv4hycuAdyQ5mt71AWcftPmrugsrHwPuBP4SWDlj/TuBP01yG/AocGFVfeOgw+TLTlXdkuTTwAVV9Z4kzwKu7z73w8DPVNUdSd4CfCzJY/QOj18IvAm4Msleev84nDyKz7DUJbkHeBpwZJLNwDlVdedoq9Kg2b+Gwx42Wsuhf/lVLpIkSY0Ol1N4kiRJA2OAkiRJamSAkiRJamSAkiRJamSAkiRJamSAkiRJamSAkiRJavT/A/K2n52A99kDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = scorer.scores['ents_per_type']\n",
    "overall = scorer.scores\n",
    "f, p, r = overall['ents_f'], overall['ents_p'], overall['ents_r']\n",
    "x = ['Precision', 'Recall', 'F1']\n",
    "\n",
    "fig, ax = plt.subplots(2, 2, figsize=(10, 10), sharey=True, \n",
    "                       subplot_kw={'xticklabels': x}\n",
    "                      )\n",
    "ax = ax.flatten()\n",
    "\n",
    "for a, tag in zip(ax, ['PERSON', 'LOC', 'ORG', 'OVERALL']):\n",
    "    \n",
    "    if tag == 'OVERALL':\n",
    "        y = [f, p, r]\n",
    "    else: \n",
    "        y = list(scores[tag].values())\n",
    "    \n",
    "    sns.barplot(x=x, y=y, ax=a, palette='GnBu', edgecolor='black')\n",
    "    a.set_title(str.title(tag))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary\n",
    "This dataset is a combination of the Emerging Entities data and the CONLL2003 data (and CONLL2003, the easier dataset, represents over 80% of the entities). \n",
    "\n",
    "Given that SoTA on CONLL2003 is over 95 at this point, and this is spaCy's best English model, AND that the model is trained on this specific genre of text (news and blogs), these numbers are surprising.\n",
    "\n",
    "One thing that seems particularly odd is the extremely low recall and F1 for LOC, which is not an especially difficult tag in the NER space. I'd be curious to know if there's some kind of quirk in the scoring function that's causing this - spaCy uses BILUO tags rather than BIO tags (and exact match for scoring), so this could be too strict a criteria. \n",
    "\n",
    "SpaCy's reported scores for this model:\n",
    "\n",
    "NER F 86.62\n",
    "\n",
    "NER P 87.03\n",
    "\n",
    "NER R 86.20\n",
    "\n",
    "The spaCy model wasn't trained with a MISC tag, so info on the performance there is not available.\n",
    "\n",
    "Matt Honnibal weighs in on this [here](https://github.com/explosion/spaCy/issues/2169), but he mainly talks about superficial differences (differences in labeling scheme), which have already been addressed here. He claims that it is possible to get around 92% on CONLL using the spacy model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
